 
  

 






<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> 
<html>

<!-- Mirrored from www.javapractices.com/topic/TopicAction.do;jsessionid=4FCCB481C702D708A7360133D128E359?Id=87 by HTTrack Website Copier/3.x [XR&CO'2010], Sun, 12 Jun 2011 17:27:37 GMT -->
<!-- Added by HTTrack --><meta http-equiv="content-type" content="text/html;charset=UTF-8"><!-- /Added by HTTrack -->
<head>
 <title>
  Java Practices -> Parse text
 </title>
 <link rel="stylesheet" type="text/css" href="../stylesheet8.css" media="all">
 
 <link rel="shortcut icon" href='../images/favicon.ico' type="image/vnd.microsoft.icon">
 <meta name="description" content="Concise presentations of java programming practices, tasks, and conventions, amply illustrated with syntax highlighted code examples.">
 
 <meta name='keywords' content='regexp,stream,streamtokenizer,stringtokenizer,token,parse,text,java,java programming,java practices,java idiom,java style,java design patterns,java coding conventions,'>
 
 
</head>
 
<body>


<div class='menu-bar'>
 
  <a href='../home/HomeAction.html' title='Table of Contents'>Home</a> |
  <a href='../vote/VoteSummaryAction-2.html' title='View Poll Results'>Poll</a> |
   
  <A href='../feedback/FeedbackAction451f-2.html?Operation=Show' title='Send Your Feedback'>Wiki</a> |
  <b><a href='../source/SourceAction-2.html' title='Grab Source Code'>Source Code</a></b><IMG class='no-margin' SRC="../images/goldstar.gif" ALT=""> |

  <a href='http://www.web4j.com/Java_Web_Application_Framework_Overview.jsp?From=1' title='Free Download - Java Web Application Framework'><b>WEB4J</b></a> |
  
  <a href='http://www.date4j.net/' title='Replacement for java.util.Date'><b>DATE4J</b></a> |

   <a href='../references/ReferencesAction-2.html' title='References'>Links</a>
   
  <form action='http://www.javapractices.com/search/SearchAction.do' method='get' class='search-form'>
   <input type='text' name='SearchTerms' value="" size=12 maxlength=50 class='search'>
   <input type='submit' value="Search">
  </form>
 
</div>

<P>



  

 






<p class="display-messages">

 

 

</p>


<div class="main-layout">
 
   

 




<div class='page-title'>Parse text</div>

<div class='main-body'>
 
<br>There are various ways of parsing text. The usual tools are :
<ul>
<li>
<a href="http://java.sun.com/javase/6/docs/api/java/lang/String.html#split(java.lang.String)">String.split</a>
methods</li>

<li>
<a href="http://java.sun.com/javase/6/docs/api/java/util/StringTokenizer.html">StringTokenizer</a>
and&nbsp; <a href="http://java.sun.com/javase/6/docs/api/java/io/StreamTokenizer.html">StreamTokenizer</a>
classes</li>

<li>
<a href="http://java.sun.com/javase/6/docs/api/java/util/Scanner.html">Scanner</a>
class</li>

<li>
<a href="http://java.sun.com/javase/6/docs/api/java/util/regex/Pattern.html">Pattern</a>
and <a href="http://java.sun.com/javase/6/docs/api/java/util/regex/Matcher.html">Matcher</a>
classes, which implement regular expressions</li>

<li>
for the most complex parsing tasks, you can use tools such as <a href="https://javacc.dev.java.net/">JavaCC</a></li>
</ul>

<b>Example 1</b>
<p>This example uses <tt>Scanner</tt>. Here, the contents of a file containing name-value 
pairs is read, and each line is parsed into its constituent data.
<br>
<PRE>

<span class='keyword'>import</span> java.io.*;
<span class='keyword'>import</span> java.util.Scanner;

<span class='keyword'>public</span> <span class='keyword'>class</span> ReadWithScanner {

  <span class='keyword'>public</span> <span class='keyword'>static</span> <span class='keyword'>void</span> main(String... aArgs) <span class='keyword'>throws</span> FileNotFoundException {
    ReadWithScanner parser = <span class='keyword'>new</span> ReadWithScanner(<span class='literal'>"C:&#092;&#092;Temp&#092;&#092;test.txt"</span>);
    parser.processLineByLine();
    log(<span class='literal'>"Done."</span>);
  }
  
  <span class='comment'>/**
   Constructor.
   @param aFileName full name of an existing, readable file.
  */</span>
  <span class='keyword'>public</span> ReadWithScanner(String aFileName){
    fFile = <span class='keyword'>new</span> File(aFileName);  
  }
  
  <span class='comment'>/** Template method that calls {@link #processLine(String)}.  */</span>
  <span class='keyword'>public</span> <span class='keyword'>final</span> <span class='keyword'>void</span> processLineByLine() <span class='keyword'>throws</span> FileNotFoundException {
    <span class='comment'>//Note that FileReader is used, not File, since File is not Closeable
</span>    Scanner scanner = <span class='keyword'>new</span> Scanner(<span class='keyword'>new</span> FileReader(fFile));
    <span class='keyword'>try</span> {
      <span class='comment'>//first use a Scanner to get each line
</span>      <span class='keyword'>while</span> ( scanner.hasNextLine() ){
        processLine( scanner.nextLine() );
      }
    }
    <span class='keyword'>finally</span> {
      <span class='comment'>//ensure the underlying stream is always closed
</span>      <span class='comment'>//this only has any effect if the item passed to the Scanner
</span>      <span class='comment'>//constructor implements Closeable (which it does in this case).
</span>      scanner.close();
    }
  }
  
  <span class='comment'>/** 
   Overridable method for processing lines in different ways.
    
   &lt;P&gt;This simple default implementation expects simple name-value pairs, separated by an 
   '=' sign. Examples of valid input : 
   &lt;tt&gt;height = 167cm&lt;/tt&gt;
   &lt;tt&gt;mass =  65kg&lt;/tt&gt;
   &lt;tt&gt;disposition =  "grumpy"&lt;/tt&gt;
   &lt;tt&gt;this is the name = this is the value&lt;/tt&gt;
  */</span>
  <span class='keyword'>protected</span> <span class='keyword'>void</span> processLine(String aLine){
    <span class='comment'>//use a second Scanner to parse the content of each line 
</span>    Scanner scanner = <span class='keyword'>new</span> Scanner(aLine);
    scanner.useDelimiter(<span class='literal'>"="</span>);
    <span class='keyword'>if</span> ( scanner.hasNext() ){
      String name = scanner.next();
      String value = scanner.next();
      log(<span class='literal'>"Name is : "</span> + quote(name.trim()) + <span class='literal'>", and Value is : "</span> + quote(value.trim()) );
    }
    <span class='keyword'>else</span> {
      log(<span class='literal'>"Empty or invalid line. Unable to process."</span>);
    }
    <span class='comment'>//no need to call scanner.close(), since the source is a String
</span>  }
  
  <span class='comment'>// PRIVATE 
</span>  <span class='keyword'>private</span> <span class='keyword'>final</span> File fFile;
  
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>void</span> log(Object aObject){
    System.out.println(String.valueOf(aObject));
  }
  
  <span class='keyword'>private</span> String quote(String aText){
    String QUOTE = <span class='literal'>"'"</span>;
    <span class='keyword'>return</span> QUOTE + aText + QUOTE;
  }
} 
</PRE>
<br>
<br>For a file containing :
<PRE>
height = 167cm
mass =  65kg
disposition =  "grumpy"
this is the name = this is the value
</PRE>
the output of the above class is :
<PRE>
Name is : 'height', and Value is : '167cm'
Name is : 'mass', and Value is : '65kg'
Name is : 'disposition', and Value is : '"grumpy"'
Name is : 'this is the name', and Value is : 'this is the value'
Done.
</PRE>

<P><b>Example 2</b>
<p>This example uses <tt>StringTokenizer</tt>. 
This class is used to parse the text entered into a search box on a web page. It 
returns a <tt>Set</tt> of tokens to be used for pattern
matching. Here, any text appearing in quotes is treated as a single search token. All
other text is split into tokens based simply on whitespace.
<p>An example run:
<p><tt>>java -cp . SearchBoxParser</tt>
<br><tt>[mars, sun, milky way, venus]</tt>
<br>
<PRE>

<span class='keyword'>import</span> java.util.*;

<span class='comment'>/**
* The user enters text into a search box. This class is used
* to parse that text into specific search terms (or tokens).
* It eliminates common words, and allows for the quoting of text, using
* double quotes.
*/</span>
<span class='keyword'>public</span> <span class='keyword'>final</span> <span class='keyword'>class</span> SearchBoxParser {

  <span class='keyword'>public</span> <span class='keyword'>static</span> <span class='keyword'>void</span> main(String... aArguments) {
    SearchBoxParser parser = <span class='keyword'>new</span> SearchBoxParser( <span class='literal'>"mars venus &#092;"milky way&#092;" sun"</span> );
    Set&lt;String&gt; tokens = parser.parseSearchText();
    <span class='comment'>//display the tokens
</span>    System.out.println(tokens);
  }

  <span class='comment'>/**
  * @param aSearchText is non-null, but may have no content,
  * and represents what the user has input in a search box.
  */</span>
  <span class='keyword'>public</span> SearchBoxParser( String aSearchText ) {
    <span class='keyword'>if</span> ( aSearchText == <span class='keyword'>null</span> ) {
      <span class='keyword'>throw</span> <span class='keyword'>new</span> IllegalArgumentException(<span class='literal'>"Search Text cannot be null."</span>);
    }
    fSearchText = aSearchText;
  }

  <span class='comment'>/**
  * Parse the user's search box input into a Set of String tokens.
  *
  * @return Set of Strings, one for each word in fSearchText; here "word"
  * is defined as either a lone word surrounded by whitespace, or as a series
  * of words surrounded by double quotes, "like this"; also, very common
  * words (and, the, etc.) do not qualify as possible search targets.
  */</span>
  <span class='keyword'>public</span> Set&lt;String&gt; parseSearchText() {
    Set&lt;String&gt; result = <span class='keyword'>new</span> HashSet&lt;String&gt;();

    <span class='keyword'>boolean</span> returnTokens = <span class='keyword'>true</span>;
    String currentDelims = fWHITESPACE_AND_QUOTES;
    StringTokenizer parser = <span class='keyword'>new</span> StringTokenizer(
      fSearchText,
      currentDelims,
      returnTokens
    );

    String token = <span class='keyword'>null</span>;
    <span class='keyword'>while</span> ( parser.hasMoreTokens() ) {
      token = parser.nextToken(currentDelims);
      <span class='keyword'>if</span> ( !isDoubleQuote(token) ){
        addNonTrivialWordToResult( token, result );
      }
      <span class='keyword'>else</span> {
        currentDelims = flipDelimiters(currentDelims);
      }
    }
    <span class='keyword'>return</span> result;
  }

  <span class='comment'>// PRIVATE //
</span>  <span class='keyword'>private</span> String fSearchText;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> Set&lt;String&gt; fCOMMON_WORDS = <span class='keyword'>new</span> HashSet&lt;String&gt;();
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String fDOUBLE_QUOTE = <span class='literal'>"&#092;""</span>;

  <span class='comment'>//the parser flips between these two sets of delimiters
</span>  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String fWHITESPACE_AND_QUOTES = <span class='literal'>" &#092;t&#092;r&#092;n&#092;""</span>;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String fQUOTES_ONLY =<span class='literal'>"&#092;""</span>;

  <span class='comment'>/**
  * Very common words against which searches will not be
  * performed.
  */</span>
  <span class='keyword'>static</span> {
    fCOMMON_WORDS.add(<span class='literal'>"a"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"and"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"be"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"for"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"from"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"has"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"i"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"in"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"is"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"it"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"of"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"on"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"to"</span>);
    fCOMMON_WORDS.add(<span class='literal'>"the"</span>);
  }

  <span class='comment'>/**
  * Use to determine if a particular word entered in the
  * search box should be discarded from the search.
  */</span>
  <span class='keyword'>private</span> <span class='keyword'>boolean</span> isCommonWord( String aSearchTokenCandidate ) {
    <span class='keyword'>return</span> fCOMMON_WORDS.contains(aSearchTokenCandidate);
  }

  <span class='keyword'>private</span> <span class='keyword'>boolean</span> textHasContent(String aText) {
    <span class='keyword'>return</span> (aText != <span class='keyword'>null</span>) &amp;&amp; (!aText.trim().equals(<span class='literal'>""</span>));
  }

  <span class='keyword'>private</span> <span class='keyword'>void</span> addNonTrivialWordToResult( String aToken, Set&lt;String&gt; aResult ){
    <span class='keyword'>if</span> ( textHasContent(aToken) &amp;&amp; !isCommonWord(aToken.trim()) ) {
      aResult.add( aToken.trim() );
    }
  }

  <span class='keyword'>private</span> <span class='keyword'>boolean</span> isDoubleQuote( String aToken ){
    <span class='keyword'>return</span> aToken.equals(fDOUBLE_QUOTE);
  }

  <span class='keyword'>private</span> String flipDelimiters( String aCurrentDelims ) {
    String result = <span class='keyword'>null</span>;
    <span class='keyword'>if</span> ( aCurrentDelims.equals(fWHITESPACE_AND_QUOTES) ) {
      result = fQUOTES_ONLY;
    }
    <span class='keyword'>else</span> {
      result = fWHITESPACE_AND_QUOTES;
    }
    <span class='keyword'>return</span> result;
  }
} 
</PRE>
<br>

<br><b>Example 3</b>
<p>This example demonstrates use of regular expressions, by parsing a fully-qualified
type name into two parts - the package and the "simple" type name.
<br>
<PRE>

<span class='keyword'>import</span> java.util.regex.*;

<span class='keyword'>public</span> <span class='keyword'>final</span> <span class='keyword'>class</span> RegularExpressions {

  <span class='comment'>/**
  * The pattern is matched to the first argument.
  */</span>
  <span class='keyword'>public</span> <span class='keyword'>static</span> <span class='keyword'>void</span> main (String[] aArguments) {
    matchParts(aArguments[<span class='literal'>0</span>]);
    matchAll(aArguments[<span class='literal'>0</span>]);
  }

  <span class='comment'>/**
  * The Matcher.find method attempts to match *parts* of the input
  * to the given pattern.
  */</span>
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>void</span> matchParts( String aText ){
    System.out.println(fNEW_LINE + <span class='literal'>"Match PARTS:"</span>);
    <span class='comment'>//(note the necessity of the comments flag, since our regular
</span>    <span class='comment'>//expression contains comments:)
</span>    Pattern pattern = Pattern.compile( fREGEXP, Pattern.COMMENTS );
    Matcher matcher = pattern.matcher( aText );
    <span class='keyword'>while</span> ( matcher.find() ) {
      System.out.println(<span class='literal'>"Num groups: "</span> + matcher.groupCount());
      System.out.println(<span class='literal'>"Package: "</span> + matcher.group(<span class='literal'>1</span>));
      System.out.println(<span class='literal'>"Class: "</span> + matcher.group(<span class='literal'>2</span>));
    }
  }

  <span class='comment'>/**
  * The Matcher.matches method attempts to match the *entire*
  * input to the given pattern all at once.
  */</span>
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>void</span> matchAll( String aText ){
    System.out.println(fNEW_LINE + <span class='literal'>"Match ALL:"</span>);
    Pattern pattern = Pattern.compile( fREGEXP, Pattern.COMMENTS );
    Matcher matcher = pattern.matcher( aText );
    <span class='keyword'>if</span>( matcher.matches() ) {
      System.out.println(<span class='literal'>"Num groups: "</span> + matcher.groupCount());
      System.out.println(<span class='literal'>"Package: "</span> + matcher.group(<span class='literal'>1</span>));
      System.out.println(<span class='literal'>"Class: "</span> + matcher.group(<span class='literal'>2</span>));
    }
    <span class='keyword'>else</span> {
      System.err.println(<span class='literal'>"Input does not match pattern."</span>);
    }
  }

  <span class='comment'>//PRIVATE //
</span>
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String fNEW_LINE = System.getProperty(<span class='literal'>"line.separator"</span>);

  <span class='comment'>/**
  * A commented regular expression for fully-qualified type names which
  * follow the common naming conventions, for example, "com.myappBlah.Thing".
  *
  * Thus, the "dot + capital letter" is sufficient to define where the
  * package names end.
  *
  * This regular expression uses two groups, one for the package, and one
  * for the class. Groups are defined by parentheses. Note that ?: will
  * define a group as "non-contributing"; that is, it will not contribute
  * to the return values of the &lt;code&gt;group&lt;/code&gt; method.
  */</span>
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String fREGEXP =
      <span class='literal'>"#Group1 - Package prefix without last dot: "</span> + fNEW_LINE +
      <span class='literal'>"( (?:&#092;&#092;w|&#092;&#092;.)+ ) &#092;&#092;."</span> + fNEW_LINE +
      <span class='literal'>"#Group2 - Class name starts with uppercase: "</span> + fNEW_LINE +
      <span class='literal'>"( [A-Z](?:&#092;&#092;w)+ )"</span>;
} 
</PRE>
<br>
<br>Some example runs:
<p><tt>>java -cp . RegularExpressions "java.java.Thing java.lang.Random"</tt>
<p><tt>Match PARTS:</tt>
<br><tt>Num groups: 2</tt>
<br><tt>Package: java.java</tt>
<br><tt>Class: Thing</tt>
<br><tt>Num groups: 2</tt>
<br><tt>Package: java.lang</tt>
<br><tt>Class: Random</tt>
<p><tt>Match ALL:</tt>
<br><tt>Input does not match pattern.</tt>
<p><tt>>java -cp . RegularExpressions "java.java.Thing"</tt>
<p><tt>Match PARTS:</tt>
<br><tt>Num groups: 2</tt>
<br><tt>Package: java.java</tt>
<br><tt>Class: Thing</tt>
<p><tt>Match ALL:</tt>
<br><tt>Num groups: 2</tt>
<br><tt>Package: java.java</tt>
<br><tt>Class: Thing</tt>
<p><tt>>java -cp . RegularExpressions "java.java.Thing "</tt>
<p><tt>Match PARTS:</tt>
<br><tt>Num groups: 2</tt>
<br><tt>Package: java.java</tt>
<br><tt>Class: Thing</tt>
<p><tt>Match ALL:</tt>
<br><tt>Input does not match pattern.</tt>

<p>
<br>

</div>




<div class='topic-section'>See Also :</div>
<div class='main-body'>
 
  
  <a href='TopicAction12d8-2.html?Id=42'>Reading and writing text files</a> <br>
 
  
  <a href='TopicAction3c83-2.html?Id=103'>Pattern-match lines of a file</a> <br>
 
  
  <a href='TopicActionb2b7-2.html?Id=104'>Compile regular expressions once</a> <br>
 
</div>


<div class='topic-section'>Would you use this technique?</div>
<div class='main-body'>
  
  <form action="http://www.javapractices.com/vote/AddVoteAction.do" method='post'>
    Yes<input type='radio' name='Choice' value='Y' >
    &nbsp;&nbsp;No<input type='radio' name='Choice' value='N'>
    &nbsp;&nbsp;Undecided<input type='radio' name='Choice' value="?" >
    &nbsp;&nbsp;<input type=submit value="Vote" >
    <input type='hidden' name='Operation' value='Apply'>
    <input type='hidden' name='TopicId' value='87'>
  </form>
</div>

<div style='height:10.0em;'></div>

 
 
</div>

  

 





<div align='center' class='legalese'>  
&copy; 2011 Hirondelle Systems |
<a href='../source/SourceAction-2.html'><b>Source Code</b></a><IMG class='no-margin' SRC="../images/goldstar.gif" ALT=""> |
<a href="mailto:webmaster@javapractices.com">Contact</a> |
<a href="http://creativecommons.org/licenses/by-nc-sa/1.0/">License</a> |
<a href='../apps/cjp.rss'>RSS</a>
<!-- ukey="2AC36CD2" -->
<!-- ckey="16DF3D87" -->
<br>

 Individual code snippets can be used under this <a href='../LICENSE.txt'>BSD license</a> - Last updated on June 6, 2010.<br>
 Over 150,000 unique IPs last month - <span title='Java Practices 2.6.5, Mon May 16 00:00:00 EDT 2011'>Built with</span> <a href='http://www.web4j.com/'>WEB4J</a>.<br>
 - In Memoriam : Bill Dirani -
</div>

<script src="../../www.google-analytics.com/urchin.js" type="text/javascript">
</script>
<script type="text/javascript">
_uacct = "UA-2633428-1";
urchinTracker();
</script>



</body>

<!-- Mirrored from www.javapractices.com/topic/TopicAction.do;jsessionid=4FCCB481C702D708A7360133D128E359?Id=87 by HTTrack Website Copier/3.x [XR&CO'2010], Sun, 12 Jun 2011 17:27:37 GMT -->
<!-- Added by HTTrack --><meta http-equiv="content-type" content="text/html;charset=UTF-8"><!-- /Added by HTTrack -->
</html>
